
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import re
import folium
from wordcloud import WordCloud
import matplotlib.colors as mcolors
import branca
from sklearn.cluster import KMeans
from sklearn.preprocessing import StandardScaler
import geopandas as gpd
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error, r2_score
from sklearn.tree import DecisionTreeRegressor


data = pd.read_csv('rightmove.csv')
len(data)
print(len(data))
data.describe()  # Get summary statistics for numerical columns
data.info()  # Get summary statistics for numerical columns
data.head()# Get summary statistics for numerical columns
print(data.head())
data.tail()  # Get summary statistics for numerical columns
data['price'].describe()
print(data['price'].describe())

24899
<class 'pandas.core.frame.DataFrame'>
RangeIndex: 24899 entries, 0 to 24898
Data columns (total 8 columns):
 #   Column         Non-Null Count  Dtype  
---  ------         --------------  -----  
 0   title          24899 non-null  object 
 1   property_type  24899 non-null  object 
 2   price          24899 non-null  object 
 3   house_area     6884 non-null   object 
 4   address        24899 non-null  object 
 5   Feature        24899 non-null  object 
 6   latitude       24655 non-null  float64
 7   longitude      24655 non-null  float64
dtypes: float64(2), object(6)
memory usage: 1.5+ MB
                 title property_type       price house_area  \
0  1 bedroom apartment     Apartment     595,000        NaN   
1       4 bedroom flat          Flat  64,999,999        NaN   
2  5 bedroom apartment     Apartment  60,000,000        NaN   
3      7 bedroom house         House  45,000,000      1,247   
4     12 bedroom house         House  45,000,000      1,360   

                                       address  \
0                    Kilburn Lane, London, W10   
1                  Knightsbridge, London, SW1X   
2                 One Hyde Park, Knightsbridge   
3                 Lygon Place, Belgravia, SW1W   
4  Mayfair Freehold House, Park Lane Area, W1K   

                                             Feature   latitude  longitude  
0  ['Private terrace', 'Communcal terrace', 'Bout...  51.532046  -0.210721  
1  ['Super Prime', 'London', 'Knightsbridge and M...  51.497825  -0.154109  
2  ['Apartment', '5 Beds', '2 Receptions', '5 Bat...  51.501984  -0.160544  
3  ['Cinema Room', 'Entertainment Room', 'CCTV', ...  51.496589  -0.146982  
4  ['Mayfair mansion with 26 meter frontage', 'Lo...  51.508009  -0.154332  
count         24899
unique          475
top       6,000,000
freq           1221
Name: price, dtype: object


data = data.dropna()  # Remove rows with missing values
property_counts = data['property_type'].value_counts()

# Plot the bar chart
plt.figure(figsize=(20, 6))
sns.barplot(x=property_counts.index, y=property_counts.values)
plt.xlabel('Property Type')
plt.ylabel('Count')
plt.title('Number of Properties by Type')
plt.show()


data['price'] = pd.to_numeric(data['price'].astype(str).str.replace(',', ''), errors='coerce')
data['house_area'] = pd.to_numeric(data['house_area'].astype(str).str.replace(',', ''), errors='coerce')


# Remove rows with non-numeric values (such as 'POA') in the 'price' column
data.dropna(subset=['price'], inplace=True)
plt.figure(figsize=(12, 6))
sns.histplot(data['price'], kde=True, bins=50)
plt.xlabel('Price')
plt.ylabel('Density')
plt.title('Price Distribution with Kernel Density Estimation')
plt.show()
# Define the price range bins
bins = [0, 10000000, 20000000, 30000000, 40000000, float('inf')]
labels = ['100k-10M', '10M-20M','20M-30M', '30M-40M', '40M+']
bins = np.sort(np.unique(bins))
# Categorize properties into price range bins
data['price_range'] = pd.cut(data['price'], bins=bins, labels=labels)

# Count the number of properties in each bin
property_counts = data['price_range'].value_counts().sort_index()

# Plot the bar chart
plt.figure(figsize=(12, 6))
sns.barplot(x=property_counts.index, y=property_counts.values)
plt.xlabel('Price Range')
plt.ylabel('Count')
plt.title('Number of Properties by Price Range')
plt.xticks(rotation=45)
plt.show()


from folium.plugins import HeatMap
london = gpd.read_file(gpd.datasets.get_path('naturalearth_lowres')).query('name == "United Kingdom"')
mean_lat, mean_lon = data['latitude'].mean(), data['longitude'].mean()
std_lat, std_lon = data['latitude'].std(), data['longitude'].std()

# Set the threshold for removing outliers (e.g., 3 times the standard deviation)
threshold = 3

# Filter out points that are far away from the mean
filtered_data = data[((data['latitude'] - mean_lat).abs() <= threshold * std_lat) &
                     ((data['longitude'] - mean_lon).abs() <= threshold * std_lon)]

# Calculate the center of the map (use the mean latitude and longitude)
center_lat, center_lon = filtered_data['latitude'].mean(), filtered_data['longitude'].mean()

# Create a Folium map
map = folium.Map(location=[center_lat, center_lon], zoom_start=11)
data_heatmap = []
# Add circle markers to the map
for _, row in filtered_data.iterrows():
    folium.CircleMarker(location=[row['latitude'], row['longitude']],
                        color='blue',
                        fill=True,
                        fill_color='blue',
                        fill_opacity=0.8,
                        radius=1).add_to(map)
    data_heatmap.append([row['latitude'], row['longitude']])
# Display the map
HeatMap(data_heatmap).add_to(map)
map


# Calculate the center of the map (use the mean latitude and longitude)
center_lat, center_lon = filtered_data['latitude'].mean(), filtered_data['longitude'].mean()

# Create a Folium map
property_type_map = folium.Map(location=[center_lat, center_lon], zoom_start=11)

# Define the color mapping based on property_type
unique_property_types = filtered_data['property_type'].unique()
n_unique_property_types = len(unique_property_types)

# Generate a color palette with the same number of colors as unique property types
color_palette = sns.color_palette('husl', n_unique_property_types)

# Convert the colors from RGB to hexadecimal
hex_colors = [mcolors.rgb2hex(color) for color in color_palette]

# Create a dictionary mapping property types to colors
property_type_colors = {property_type: color for property_type, color in zip(unique_property_types, hex_colors)}

# Define a function to map hex colors to folium colors
def map_hex_to_folium_color(hex_color):
    folium_colors = ['red', 'blue', 'green', 'purple', 'orange', 'darkred', 'lightred', 'beige', 'darkblue', 'darkgreen', 'cadetblue', 'darkpurple', 'white', 'pink', 'lightblue', 'lightgreen', 'gray', 'black', 'lightgray']
    return folium_colors[hex_colors.index(hex_color)]

# Add the property locations to the map with different colors based on property_type
for _, row in filtered_data.iterrows():
    folium.CircleMarker(location=[row['latitude'], row['longitude']],
                        radius=1, # Adjust the size of the circle by changing the radius value
                        color=property_type_colors[row['property_type']],
                        fill=True,
                        fill_color=property_type_colors[row['property_type']],
                        fill_opacity=1).add_to(property_type_map)
# Create an HTML legend
legend_html = '''
<div style="position: fixed;
            bottom:  250px; left: 50px; width: 150px; height: 150px;
            border:2px solid grey; z-index:9999; font-size:14px;">
    &nbsp;<b>Property Type</b><br>
'''

for property_type, color in property_type_colors.items():
    legend_html += f'<i style="background:{color}; width: 10px; height: 10px; display: inline-block; margin-left: 5px;"></i> {property_type}<br>'

legend_html += '</div>'

# Add the legend to the map
legend = branca.element.Element(legend_html)
property_type_map.get_root().html.add_child(legend)

# Display the map
property_type_map


center_lat, center_lon = filtered_data['latitude'].mean(), filtered_data['longitude'].mean()

# Create a Folium map
price_range_map = folium.Map(location=[center_lat, center_lon], zoom_start=11)

# Define the color mapping based on property_type
unique_price_range = filtered_data['price_range'].unique()
n_unique_price_range = len(unique_price_range)

# Generate a color palette with the same number of colors as unique property types
color_palette = sns.color_palette('husl', n_unique_price_range)

# Convert the colors from RGB to hexadecimal
hex_colors = [mcolors.rgb2hex(color) for color in color_palette]

# Create a dictionary mapping property types to colors
price_range_colors = {price_range: color for price_range, color in zip(unique_price_range, hex_colors)}

# Define a function to map hex colors to folium colors
def map_hex_to_folium_color(hex_color):
    folium_colors = ['red', 'blue', 'green', 'purple', 'orange', 'darkred', 'lightred', 'beige', 'darkblue', 'darkgreen', 'cadetblue', 'darkpurple', 'white', 'pink', 'lightblue', 'lightgreen', 'gray', 'black', 'lightgray']
    return folium_colors[hex_colors.index(hex_color)]

# Add the property locations to the map with different colors based on property_type
for _, row in filtered_data.iterrows():
    folium.CircleMarker(location=[row['latitude'], row['longitude']],
                        radius=3, # Adjust the size of the circle by changing the radius value
                        color=price_range_colors[row['price_range']],
                        fill=True,
                        fill_color=price_range_colors[row['price_range']],
                        fill_opacity=1).add_to(price_range_map)
# Create an HTML legend
legend_html = '''
<div style="position: fixed;
            bottom: 50px; left: 50px; width: 150px; height: 150px;
            border:2px solid grey; z-index:9999; font-size:14px;">
    &nbsp;<b>Price Range</b><br>
'''

for price_range, color in price_range_colors.items():
    legend_html += f'<i style="background:{color}; width: 10px; height: 10px; display: inline-block; margin-left: 5px;"></i> {price_range}<br>'

legend_html += '</div>'

# Add the legend to the map
legend = branca.element.Element(legend_html)
price_range_map.get_root().html.add_child(legend)

# Display the map
price_range_map


filtered_data = filtered_data[filtered_data['Feature'] != '[]']
filtered_data['Feature'] = filtered_data['Feature'].apply(lambda x: re.sub('[^a-zA-Z0-9\s\,]+', '', x))
filtered_data['features_list'] = filtered_data['Feature'].apply(lambda x: x.split(', '))
filtered_data_high_price = filtered_data[filtered_data['price'] > 5000000]
all_features_text = " ".join(feature for feature_list in filtered_data_high_price['features_list'] for feature in feature_list)
wordcloud = WordCloud(width=800, height=800, background_color='white', min_font_size=10).generate(all_features_text)
plt.figure(figsize=(8, 8), facecolor=None)
plt.imshow(wordcloud)
plt.axis("off")
plt.tight_layout(pad=0)
plt.show()


filtered_data_low_price = filtered_data[(filtered_data['price'] <5000000)]
all_features_text = " ".join(feature for feature_list in filtered_data_low_price['features_list'] for feature in feature_list)
wordcloud = WordCloud(width=800, height=800, background_color='white', min_font_size=10).generate(all_features_text)
plt.figure(figsize=(8, 8), facecolor=None)
plt.imshow(wordcloud)
plt.axis("off")
plt.tight_layout(pad=0)
plt.show()


filtered_data['house_area'] = pd.to_numeric(filtered_data['house_area'], errors='coerce')
filtered_data['price_per_sqft'] = filtered_data['price'] / filtered_data['house_area']
filtered_data['price_per_sqft'].describe()
print(filtered_data['price_per_sqft'].describe())

count     5055.000000
mean     30757.709101
std      12280.881988
min         89.160272
25%      22577.519380
50%      30252.100840
75%      38390.313390
max      79881.656805
Name: price_per_sqft, dtype: float64


filtered_data = filtered_data.dropna(subset=['price_per_sqft'])
price_per_sqft_scaled = StandardScaler().fit_transform(filtered_data[['price_per_sqft']])

# Determine the range of possible cluster numbers
range_n_clusters = range(1, 11)

# Calculate the average distortion for each number of clusters
distortions = []
for n_clusters in range_n_clusters:
    kmeans = KMeans(n_clusters=n_clusters, random_state=42)
    kmeans.fit(price_per_sqft_scaled)
    distortions.append(kmeans.inertia_)

# Plot the elbow curve
plt.figure(figsize=(8, 6))
plt.plot(range_n_clusters, distortions, marker='o')
plt.xlabel('Number of clusters')
plt.ylabel('Distortion')
plt.title('Elbow curve for price_per_sqft clustering')
plt.show()


filtered_data = filtered_data.dropna(subset=['price_per_sqft'])
scaler = StandardScaler()
price_per_sqft_scaled = scaler.fit_transform(filtered_data[['price_per_sqft']])
k = 3
kmeans = KMeans(n_clusters=k, random_state=42)
kmeans.fit(price_per_sqft_scaled)

# Add the cluster labels to the DataFrame
filtered_data['cluster'] = kmeans.labels_
cluster_summary = filtered_data.groupby('cluster')['price_per_sqft'].mean()
print(cluster_summary)
from sklearn.metrics import silhouette_score, calinski_harabasz_score

# 轮廓系数 (Silhouette Coefficient)
silhouette = silhouette_score(price_per_sqft_scaled, filtered_data['cluster'])
print(f"Silhouette Coefficient: {silhouette}")

# Calinski-Harabasz 指数 (Variance Ratio Criterion)
calinski_harabasz = calinski_harabasz_score(price_per_sqft_scaled, filtered_data['cluster'])
print(f"Calinski-Harabasz Index: {calinski_harabasz}")

cluster_range = range(2, 11)  # 聚类数量的范围，这里我们选择从 2 到 10
silhouette_scores = []

for k in cluster_range:
    kmeans = KMeans(n_clusters=k, random_state=42)
    kmeans.fit(price_per_sqft_scaled)
    cluster_labels = kmeans.labels_
    silhouette = silhouette_score(price_per_sqft_scaled, cluster_labels)
    silhouette_scores.append(silhouette)

# 绘制轮廓系数图
plt.figure(figsize=(10, 6))
plt.plot(cluster_range, silhouette_scores, marker='o')
plt.xlabel('Number of Clusters')
plt.ylabel('Silhouette Coefficient')
plt.title('Silhouette Coefficient for Different Cluster Numbers')
plt.grid()
plt.show()

cluster
0    32825.070562
1    17336.221400
2    47547.863689
Name: price_per_sqft, dtype: float64
Silhouette Coefficient: 0.5588607913977425
Calinski-Harabasz Index: 11332.929212408557


import matplotlib
k=3
# Calculate the center of the map (use the mean latitude and longitude)
center_lat, center_lon = filtered_data['latitude'].mean(), filtered_data['longitude'].mean()

# Create a Folium map
cluster_map = folium.Map(location=[center_lat, center_lon], zoom_start=11)

# Generate a color palette with the same number of colors as clusters
color_palette = sns.color_palette('viridis', k)

# Convert colors to hex format
hex_colors = [matplotlib.colors.to_hex(color) for color in color_palette]

# Create a dictionary mapping cluster labels to colors
cluster_colors = {cluster_label: color for cluster_label, color in zip(range(k), hex_colors)}

# Add circle markers to the map with different colors based on the cluster label
for _, row in filtered_data.iterrows():
    folium.CircleMarker(location=[row['latitude'], row['longitude']],
                        color=cluster_colors[row['cluster']],
                        fill=True,
                        fill_color=cluster_colors[row['cluster']],
                        fill_opacity=0.8,
                        radius=1).add_to(cluster_map)

legend_html = '''
<div style="position: fixed;
            bottom: 50px; left: 50px; width: 150px; height: 150px;
            border:2px solid grey; z-index:9999; font-size:14px;">
    &nbsp;<b>cluster result</b><br>
'''

for cluster_label, color in cluster_colors.items():
    legend_html += f'<i style="background:{color}; width: 10px; height: 10px; display: inline-block; margin-left: 5px;"></i> {cluster_label}<br>'

legend_html += '</div>'

# Add the legend to the map
legend = branca.element.Element(legend_html)
cluster_map.get_root().html.add_child(legend)
# Display the map
cluster_map


import folium
from folium.plugins import HeatMap

# Calculate the center of the map (use the mean latitude and longitude)
center_lat, center_lon = filtered_data['latitude'].mean(), filtered_data['longitude'].mean()

# Create a Folium map
cluster_heatmap = folium.Map(location=[center_lat, center_lon], zoom_start=11)

# Loop through each cluster
for cluster_label in range(k):
    # Filter the data for the current cluster
    cluster_data = filtered_data[filtered_data['cluster'] == cluster_label]
    # Prepare data for the heatmap
    data_heatmap = []
    for _, row in cluster_data.iterrows():
        data_heatmap.append([row['latitude'], row['longitude']])

    # Add a HeatMap layer to the map for the current cluster
    HeatMap(data_heatmap, name=f"Cluster {cluster_label}").add_to(cluster_heatmap)

# Add layer control to the map
folium.LayerControl().add_to(cluster_heatmap)

# Display the map
cluster_heatmap


import googlemaps
gmaps = googlemaps.Client(key='AIzaSyCCvPdzRUuGTojTUMlJh2mTa_youryBODM')
def get_nearest_school_distance(location):
    try:
        result = gmaps.places_nearby(location=location, type='school', rank_by='distance')
        if len(result['results']) > 0:
            nearest_school_location = result['results'][0]['geometry']['location']
            distance = gmaps.distance_matrix(location, nearest_school_location, units='metric')['rows'][0]['elements'][0]['distance']['value']
            return distance
        else:
            return np.nan
    except Exception as e:
        print(e)
        return np.nan
filtered_data['nearest_school_distance'] = filtered_data.apply(lambda row: get_nearest_school_distance((row['latitude'], row['longitude'])), axis=1)


def get_nearest_park_distance(location):
    try:
        result = gmaps.places_nearby(location=location, type='park', rank_by='distance')
        if len(result['results']) > 0:
            nearest_park_location = result['results'][0]['geometry']['location']
            distance = gmaps.distance_matrix(location, nearest_park_location, units='metric')['rows'][0]['elements'][0]['distance']['value']
            return distance
        else:
            return np.nan
    except Exception as e:
        print(e)
        return np.nan
filtered_data['nearest_park_distance'] = filtered_data.apply(lambda row: get_nearest_park_distance((row['latitude'], row['longitude'])), axis=1)


def extract_bedrooms(title):
    first_word = title.split()[0]
    return int(first_word) if first_word.isdigit() else 0  # Change 0 to 1 if you want to consider studio as 1 bedroom

filtered_data['number_of_bedrooms'] = filtered_data['title'].apply(extract_bedrooms)
filtered_data = filtered_data[filtered_data['number_of_bedrooms'] != 0]


columns_to_analyze = ['nearest_park_distance', 'nearest_school_distance', 'house_area', 'number_of_bedrooms', 'price']
filtered_data = filtered_data[columns_to_analyze]

# Calculate the correlation matrix
correlation_matrix = filtered_data.corr()

# Visualize the correlation matrix using a heatmap
plt.figure(figsize=(10, 8))
sns.heatmap(correlation_matrix, annot=True, cmap='coolwarm', vmin=-1, vmax=1, linewidths=.5, square=True)
plt.title('Correlation Matrix')
plt.show()


sns.set()
cols = ['nearest_park_distance', 'nearest_school_distance', 'house_area', 'number_of_bedrooms', 'price']
sns.pairplot(filtered_data[cols])
plt.show();


X = filtered_data[['nearest_park_distance', 'nearest_school_distance', 'house_area','number_of_bedrooms']]
y = filtered_data['price']
# Split the data into training and testing sets (80% training and 20% testing)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)
# Create the model
model = DecisionTreeRegressor(random_state=42)

# Train the model using the training data
model.fit(X_train, y_train)
# Predict the target variable using the test data
y_pred = model.predict(X_test)

# Calculate the mean squared error and R-squared score
mse = mean_squared_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)

print(f"Mean Squared Error: {mse:.2f}")
print(f"R-squared Score: {r2:.2f}")
# Example: predict the price of a property with given distances to a park, a school, and an area
new_property_data = [[1500, 800, 80,2]]  # Replace with the actual distances and area
predicted_price = model.predict(new_property_data)
print(f"Predicted Price: {predicted_price[0]:.2f}")
errors = y_test - y_pred


# Split the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Create and train a linear regression model
model = LinearRegression()
model.fit(X_train, y_train)

# Make predictions on the test set
y_pred = model.predict(X_test)

# Evaluate the model's performance
mse = mean_squared_error(y_test, y_pred)
r2 = r2_score(y_test, y_pred)

print("Mean Squared Error:", mse)
print("R2 Score:", r2)
models = {'Decision Tree Regressor': {'model': DecisionTreeRegressor(random_state=42), 'r2': 0},
          'Linear Regression': {'model': LinearRegression(), 'r2': 0}}

predictions = {}
for model_name, model_info in models.items():
    model = model_info['model']
    model.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    predictions[model_name] = y_test - y_pred

# Plot the distribution of errors for both models in one image
plt.figure(figsize=(10, 8))
for model_name, errors in predictions.items():
    sns.distplot(errors, bins=30, hist=False, label=model_name)

plt.title('Distribution of Errors')
plt.legend()
plt.show()
predictions = {}
for model_name, model_info in models.items():
    model = model_info['model']
    model.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    predictions[model_name] = y_pred

# Plot actual and predicted values for both models in one image
plt.figure(figsize=(10, 8))
for model_name, y_pred in predictions.items():
    plt.scatter(y_test, y_pred, alpha=0.5, label=model_name)

plt.title('Actual vs Predicted Values')
plt.xlabel('Actual Values')
plt.ylabel('Predicted Values')
plt.legend()
plt.show()

for model_name, model_info in models.items():
    model = model_info['model']
    model.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    r2 = r2_score(y_test, y_pred)
    model_info['r2'] = r2
    print(f"{model_name} R2 Score: {r2:.2f}")

# Visualize the comparison
model_names = list(models.keys())
r2_scores = [model_info['r2'] for model_info in models.values()]

plt.bar(model_names, r2_scores)
plt.xlabel('Models')
plt.ylabel('R2 Score')
plt.title('Model Comparison')
plt.show()

Mean Squared Error: 1958220459808.74
R-squared Score: 0.95
Predicted Price: 1997900.00
Mean Squared Error: 41822010285505.94
R2 Score: 0.012372262554974034

/opt/anaconda3/lib/python3.9/site-packages/sklearn/base.py:450: UserWarning: X does not have valid feature names, but DecisionTreeRegressor was fitted with feature names
  warnings.warn(
/opt/anaconda3/lib/python3.9/site-packages/seaborn/distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `kdeplot` (an axes-level function for kernel density plots).
  warnings.warn(msg, FutureWarning)
/opt/anaconda3/lib/python3.9/site-packages/seaborn/distributions.py:2619: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `kdeplot` (an axes-level function for kernel density plots).
  warnings.warn(msg, FutureWarning)

Decision Tree Regressor R2 Score: 0.95
Linear Regression R2 Score: 0.01